require 'csv'

  dataFile = 'usertagpreference.csv'
  puts '[INFO] reading original csv file......'
  begin
    inFile = File.open(dataFile, 'r')
    hash = Hash.new
    num = 0
    t1 = Time.now
    GC.disable
    inFile.each_line do |line|
      num += 1
      if num % 100000 == 0
        t2 = Time.now
        puts "[DEBUG] now is working on line #{num}, time consumption #{t2-t1}"
        t1 = t2
        if num % 1000000 == 0
          t3 = Time.now
          puts "[DEBUG] GC is working"
          GC.enable
          GC.start
          GC.disable
          puts "[DEBUG] GC has end, time consumption #{Time.now-t3}"
        end
      end
      a = line.split("\t")
      a[3] = a[3].to_i
      val = hash[a[0..2]]
      if val != nil
        val[3] = val[3] + a[3]
      else
        hash[a[0..2]] = a
      end
    end
  ensure
    inFile.close
  end
  GC.enable
  array = hash.values
  array.sort
  GC.start

  GC.disable  
  puts '[INFO] writing out to CVS file......'
  CSV.open('usertagpreference_merged.csv', 'w') do |csv|
    array.each { |i| csv << i }
  end
